#!/usr/bin/python
# -*- coding:utf-8 -*-
# @author  : micah
# @time    :  
# @function: 
# @version :


import time
import requests
import pymysql
from concurrent.futures import ThreadPoolExecutor


class Baidu():
    def __init__(self):
        self.db = pymysql.connect(host='localhost', port=3305, user='root', password='123456', db='spiders10')
        self.cursor = self.db.cursor()
        self.url = 'https://talent.baidu.com/httservice/getPostListNew'
        self.headers = {
            'Referer': 'https://talent.baidu.com/jobs/social-list?search=python',
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36'
        }

    def get_data(self, page):
        data = {
            "recruitType": "SOCIAL",
            "pageSize": "10",
            "keyWord": "python",
            "curPage": page,
            "projectType": ""
        }
        response = requests.post(self.url, headers=self.headers, data=data)
        # print(response.json())

        return self.parse_data(response.json())

    def parse_data(self, response):
        list1 = []
        for node in response['data']['list']:
            # item = ()
            id = 0
            name = node['name']
            serviceCondition = node['serviceCondition']
            workPlace = node['workPlace']
            # print(item)
            list1.append((id, name, serviceCondition, workPlace))
            # print(name, serviceCondition, workPlace)
        return list1

    def save_data(self, item):
        sql = 'INSERT INTO baidu(id, name, serviceCondition, workPlace) values(%s,%s,%s,%s)'
        try:
            self.cursor.executemany(sql, item)
            self.db.commit()
            print('数据保存正常')
        except Exception as e:
            print('保存失败', e)
            self.db.rollback()

    def create_table(self):
        # 使用预处理语句创建表
        sql = """
                CREATE TABLE IF NOT EXISTS baidu(
                    id int primary key auto_increment not null,
                    name VARCHAR(255),
                    workPlace VARCHAR(255),
                    serviceCondition VARCHAR(255)
                    )
        """
        try:
            self.cursor.execute(sql)
            print("CREATE TABLE SUCCESS.")
        except Exception as e:
            print(f'CREATE TABLE FAILED,CASE{e}')

    def run(self):
        self.create_table()
        with ThreadPoolExecutor(max_workers=5) as pool:
            for i in range(1, 10):
                response = pool.submit(self.get_data, i)
                self.save_data(response.result())

        self.db.close()


if __name__ == '__main__':
    t1 = time.time()
    baidu = Baidu()
    baidu.run()
    print('总耗时：', time.time() - t1)
